{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({1: 431, 0: 1663})"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from util import file2labels\n",
    "from collections import Counter\n",
    "\n",
    "# Dev: Positive data size = 199; Negetive data size = 1895\n",
    "\n",
    "preds = file2labels(\"predictions/baseline_0.48253968253968244.txt\")\n",
    "\n",
    "Counter(preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({1: 406, 0: 1688})"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds = file2labels(\"predictions/bart_baseline_0.4925619834710743.txt\")\n",
    "\n",
    "Counter(preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({1: 193, 0: 1901})"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds = file2labels(\"predictions/bart_augment_0.5306122448979591.txt\")\n",
    "\n",
    "Counter(preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({1: 78, 0: 2016})"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds = file2labels(\"predictions/bart_augment_0.31768953068592054.txt\")\n",
    "\n",
    "Counter(preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'input_ids': tensor([[    0,  2387,   964,    32, 50264,    53,    51,  3529,   350,   171,\n",
      "         33237,     4,     2],\n",
      "        [    0,  2387,   964,    32, 50264,    53,    51,  3529,   350,   171,\n",
      "         33237,     4,     2]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
      "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}\n",
      "{'input_ids': tensor([[    0,  2387,   964,    32, 50264,    53,    51,  3529,   350,   171,\n",
      "         33237,     4,     2],\n",
      "        [    0,  2387,   964,    32, 50264,    53,    51,  3529,   350,   171,\n",
      "         33237,     4,     2]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
      "        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}\n"
     ]
    }
   ],
   "source": [
    "from transformers import BartTokenizer, BartForConditionalGeneration\n",
    "\n",
    "tokenizer = BartTokenizer.from_pretrained(\"facebook/bart-base\")\n",
    "TXT = \"My friends are <mask> but they eat too many carbs.\"\n",
    "\n",
    "model = BartForConditionalGeneration.from_pretrained(\"facebook/bart-base\")\n",
    "input_ids = tokenizer([TXT, TXT], return_tensors=\"pt\")\n",
    "print(input_ids)\n",
    "\n",
    "with tokenizer.as_target_tokenizer():\n",
    "    input_ids = tokenizer([TXT, TXT], return_tensors=\"pt\")\n",
    "    print(input_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['not', 'good', 'healthy', 'great', 'very']"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logits = model(input_ids).logits\n",
    "\n",
    "masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item()\n",
    "probs = logits[0, masked_index].softmax(dim=0)\n",
    "values, predictions = probs.topk(5)\n",
    "\n",
    "tokenizer.decode(predictions).split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': tensor([[    0,  2387,   964,    32, 50264,    53,    51,  3529,   350,   171,\n",
       "         33237,     4,     2]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer([TXT], return_tensors=\"pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[3, 3]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yes_tok = tokenizer.convert_tokens_to_ids([\"patronizing\", \"respectful\"])\n",
    "yes_tok"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['<unk>']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(3).split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(2.6303e-08, grad_fn=<SelectBackward0>),\n",
       " tensor(2.1232e-07, grad_fn=<SelectBackward0>))"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "probs[9904], probs[3084]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Map of label to numerical label:\n",
      "{'Unbalanced_power_relations': 0, 'Shallow_solution': 1, 'Presupposition': 2, 'Authority_voice': 3, 'Metaphors': 4, 'Compassion': 5, 'The_poorer_the_merrier': 6}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>par_id</th>\n",
       "      <th>art_id</th>\n",
       "      <th>text</th>\n",
       "      <th>keyword</th>\n",
       "      <th>country</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4046</td>\n",
       "      <td>@@14767805</td>\n",
       "      <td>We also know that they can benefit by receivin...</td>\n",
       "      <td>hopeless</td>\n",
       "      <td>us</td>\n",
       "      <td>[1, 0, 0, 1, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1279</td>\n",
       "      <td>@@7896098</td>\n",
       "      <td>Pope Francis washed and kissed the feet of Mus...</td>\n",
       "      <td>refugee</td>\n",
       "      <td>ng</td>\n",
       "      <td>[0, 1, 0, 0, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8330</td>\n",
       "      <td>@@17252299</td>\n",
       "      <td>Many refugees do n't want to be resettled anyw...</td>\n",
       "      <td>refugee</td>\n",
       "      <td>ng</td>\n",
       "      <td>[0, 0, 1, 0, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4063</td>\n",
       "      <td>@@3002894</td>\n",
       "      <td>\"Budding chefs , like \"\" Fred \"\" , \"\" Winston ...</td>\n",
       "      <td>in-need</td>\n",
       "      <td>ie</td>\n",
       "      <td>[1, 0, 0, 1, 1, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4089</td>\n",
       "      <td>@@25597822</td>\n",
       "      <td>\"In a 90-degree view of his constituency , one...</td>\n",
       "      <td>homeless</td>\n",
       "      <td>pk</td>\n",
       "      <td>[1, 0, 0, 0, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>988</th>\n",
       "      <td>873</td>\n",
       "      <td>@@20374243</td>\n",
       "      <td>Citing the fact that these kids who died at Go...</td>\n",
       "      <td>poor-families</td>\n",
       "      <td>sg</td>\n",
       "      <td>[1, 0, 0, 0, 1, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>989</th>\n",
       "      <td>10070</td>\n",
       "      <td>@@15573661</td>\n",
       "      <td>Fern ? ndez was a well-known philanthropist wh...</td>\n",
       "      <td>disabled</td>\n",
       "      <td>ng</td>\n",
       "      <td>[1, 0, 0, 0, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>990</th>\n",
       "      <td>6484</td>\n",
       "      <td>@@2559173</td>\n",
       "      <td>Touched much by their plight , Commanding Offi...</td>\n",
       "      <td>homeless</td>\n",
       "      <td>lk</td>\n",
       "      <td>[1, 0, 0, 0, 0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>991</th>\n",
       "      <td>6249</td>\n",
       "      <td>@@1947926</td>\n",
       "      <td>She reiterated her ministry 's commitment to p...</td>\n",
       "      <td>women</td>\n",
       "      <td>gh</td>\n",
       "      <td>[1, 0, 0, 0, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>992</th>\n",
       "      <td>5149</td>\n",
       "      <td>@@1789214</td>\n",
       "      <td>Preaching the sermon , the Dean of the St. Pet...</td>\n",
       "      <td>vulnerable</td>\n",
       "      <td>gh</td>\n",
       "      <td>[0, 0, 0, 1, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>993 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    par_id      art_id  ... country                  label\n",
       "0     4046  @@14767805  ...      us  [1, 0, 0, 1, 0, 0, 0]\n",
       "1     1279   @@7896098  ...      ng  [0, 1, 0, 0, 0, 0, 0]\n",
       "2     8330  @@17252299  ...      ng  [0, 0, 1, 0, 0, 0, 0]\n",
       "3     4063   @@3002894  ...      ie  [1, 0, 0, 1, 1, 1, 0]\n",
       "4     4089  @@25597822  ...      pk  [1, 0, 0, 0, 0, 0, 0]\n",
       "..     ...         ...  ...     ...                    ...\n",
       "988    873  @@20374243  ...      sg  [1, 0, 0, 0, 1, 0, 0]\n",
       "989  10070  @@15573661  ...      ng  [1, 0, 0, 0, 0, 0, 0]\n",
       "990   6484   @@2559173  ...      lk  [1, 0, 0, 0, 0, 1, 0]\n",
       "991   6249   @@1947926  ...      gh  [1, 0, 0, 0, 0, 0, 0]\n",
       "992   5149   @@1789214  ...      gh  [0, 0, 0, 1, 0, 0, 0]\n",
       "\n",
       "[993 rows x 6 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from dont_patronize_me import DontPatronizeMe\n",
    "import pandas as pd\n",
    "from data_loader import rebuild_raw_dataset\n",
    "\n",
    "dpm = DontPatronizeMe(\"data\", \"\")\n",
    "\n",
    "dpm.load_task1()\n",
    "dpm.load_task2()\n",
    "\n",
    "task1df = dpm.train_task1_df\n",
    "task2df = dpm.train_task2_df\n",
    "\n",
    "task2df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>par_id_x</th>\n",
       "      <th>art_id_x</th>\n",
       "      <th>keyword_x</th>\n",
       "      <th>country_x</th>\n",
       "      <th>text</th>\n",
       "      <th>label_x</th>\n",
       "      <th>orig_label</th>\n",
       "      <th>par_id_y</th>\n",
       "      <th>art_id_y</th>\n",
       "      <th>keyword_y</th>\n",
       "      <th>country_y</th>\n",
       "      <th>label_y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>33</td>\n",
       "      <td>@@8301323</td>\n",
       "      <td>disabled</td>\n",
       "      <td>pk</td>\n",
       "      <td>Arshad said that besides learning many new asp...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>33</td>\n",
       "      <td>@@8301323</td>\n",
       "      <td>disabled</td>\n",
       "      <td>pk</td>\n",
       "      <td>[1, 0, 0, 0, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>34</td>\n",
       "      <td>@@24150149</td>\n",
       "      <td>disabled</td>\n",
       "      <td>ng</td>\n",
       "      <td>Fast food employee who fed disabled man become...</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>34</td>\n",
       "      <td>@@24150149</td>\n",
       "      <td>disabled</td>\n",
       "      <td>ng</td>\n",
       "      <td>[1, 1, 0, 0, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>42</td>\n",
       "      <td>@@4591694</td>\n",
       "      <td>hopeless</td>\n",
       "      <td>jm</td>\n",
       "      <td>Vanessa had feelings of hopelessness in her fi...</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>42</td>\n",
       "      <td>@@4591694</td>\n",
       "      <td>hopeless</td>\n",
       "      <td>jm</td>\n",
       "      <td>[0, 0, 0, 0, 0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>77</td>\n",
       "      <td>@@22454828</td>\n",
       "      <td>homeless</td>\n",
       "      <td>nz</td>\n",
       "      <td>In September , Major Nottle set off on foot fr...</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>77</td>\n",
       "      <td>@@22454828</td>\n",
       "      <td>homeless</td>\n",
       "      <td>nz</td>\n",
       "      <td>[1, 1, 0, 0, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>83</td>\n",
       "      <td>@@4672144</td>\n",
       "      <td>homeless</td>\n",
       "      <td>pk</td>\n",
       "      <td>The demographics of Pakistan and India are ver...</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>83</td>\n",
       "      <td>@@4672144</td>\n",
       "      <td>homeless</td>\n",
       "      <td>pk</td>\n",
       "      <td>[1, 1, 0, 0, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>988</th>\n",
       "      <td>10445</td>\n",
       "      <td>@@3923193</td>\n",
       "      <td>refugee</td>\n",
       "      <td>gb</td>\n",
       "      <td>More than 150 volunteers spent the night in ' ...</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>10445</td>\n",
       "      <td>@@3923193</td>\n",
       "      <td>refugee</td>\n",
       "      <td>gb</td>\n",
       "      <td>[0, 1, 0, 0, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>989</th>\n",
       "      <td>10454</td>\n",
       "      <td>@@22338535</td>\n",
       "      <td>vulnerable</td>\n",
       "      <td>ie</td>\n",
       "      <td>\"\"\" We are challenged , I suggest , to turn th...</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>10454</td>\n",
       "      <td>@@22338535</td>\n",
       "      <td>vulnerable</td>\n",
       "      <td>ie</td>\n",
       "      <td>[1, 1, 0, 0, 1, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>990</th>\n",
       "      <td>10467</td>\n",
       "      <td>@@20282330</td>\n",
       "      <td>in-need</td>\n",
       "      <td>ng</td>\n",
       "      <td>\"\"\" She has one huge platform , and informatio...</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>10467</td>\n",
       "      <td>@@20282330</td>\n",
       "      <td>in-need</td>\n",
       "      <td>ng</td>\n",
       "      <td>[1, 0, 0, 0, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>991</th>\n",
       "      <td>10468</td>\n",
       "      <td>@@16753236</td>\n",
       "      <td>hopeless</td>\n",
       "      <td>in</td>\n",
       "      <td>\"\"\" Anja Ringgren Loven I ca n't find a word t...</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>10468</td>\n",
       "      <td>@@16753236</td>\n",
       "      <td>hopeless</td>\n",
       "      <td>in</td>\n",
       "      <td>[1, 0, 1, 0, 0, 1, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>992</th>\n",
       "      <td>10469</td>\n",
       "      <td>@@16779383</td>\n",
       "      <td>homeless</td>\n",
       "      <td>ie</td>\n",
       "      <td>\"\"\" Guinness World Record of 540lbs of 7-layer...</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>10469</td>\n",
       "      <td>@@16779383</td>\n",
       "      <td>homeless</td>\n",
       "      <td>ie</td>\n",
       "      <td>[1, 0, 0, 0, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>993 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    par_id_x    art_id_x  ... country_y                label_y\n",
       "0         33   @@8301323  ...        pk  [1, 0, 0, 0, 0, 0, 0]\n",
       "1         34  @@24150149  ...        ng  [1, 1, 0, 0, 0, 0, 0]\n",
       "2         42   @@4591694  ...        jm  [0, 0, 0, 0, 0, 1, 0]\n",
       "3         77  @@22454828  ...        nz  [1, 1, 0, 0, 0, 0, 0]\n",
       "4         83   @@4672144  ...        pk  [1, 1, 0, 0, 0, 0, 0]\n",
       "..       ...         ...  ...       ...                    ...\n",
       "988    10445   @@3923193  ...        gb  [0, 1, 0, 0, 0, 0, 0]\n",
       "989    10454  @@22338535  ...        ie  [1, 1, 0, 0, 1, 1, 0]\n",
       "990    10467  @@20282330  ...        ng  [1, 0, 0, 0, 0, 0, 0]\n",
       "991    10468  @@16753236  ...        in  [1, 0, 1, 0, 0, 1, 0]\n",
       "992    10469  @@16779383  ...        ie  [1, 0, 0, 0, 0, 0, 0]\n",
       "\n",
       "[993 rows x 12 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "intersect_df = pd.merge(task1df, task2df, how ='inner', on =[\"text\"])\n",
    "# intersect_df = intersect_df.drop(columns=['par_id_y', 'art_id_y', 'keyword_y', 'country_y', 'label_y'])\n",
    "# intersect_df = intersect_df.rename(columns={\"par_id_x\": \"par_id\", \"art_id_x\": \"art_id\", \"keyword_x\": \"keyword\", \"country_x\": \"country\", \"label_x\": \"label\"})\n",
    "intersect_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>par_id</th>\n",
       "      <th>art_id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>country</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>orig_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4046</td>\n",
       "      <td>@@14767805</td>\n",
       "      <td>hopeless</td>\n",
       "      <td>us</td>\n",
       "      <td>We also know that they can benefit by receivin...</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1279</td>\n",
       "      <td>@@7896098</td>\n",
       "      <td>refugee</td>\n",
       "      <td>ng</td>\n",
       "      <td>Pope Francis washed and kissed the feet of Mus...</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8330</td>\n",
       "      <td>@@17252299</td>\n",
       "      <td>refugee</td>\n",
       "      <td>ng</td>\n",
       "      <td>Many refugees do n't want to be resettled anyw...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4063</td>\n",
       "      <td>@@3002894</td>\n",
       "      <td>in-need</td>\n",
       "      <td>ie</td>\n",
       "      <td>\"Budding chefs , like \"\" Fred \"\" , \"\" Winston ...</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4089</td>\n",
       "      <td>@@25597822</td>\n",
       "      <td>homeless</td>\n",
       "      <td>pk</td>\n",
       "      <td>\"In a 90-degree view of his constituency , one...</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2089</th>\n",
       "      <td>10462</td>\n",
       "      <td>@@22092971</td>\n",
       "      <td>homeless</td>\n",
       "      <td>gh</td>\n",
       "      <td>The sad spectacle , which occurred on Saturday...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2090</th>\n",
       "      <td>10463</td>\n",
       "      <td>@@4676355</td>\n",
       "      <td>refugee</td>\n",
       "      <td>pk</td>\n",
       "      <td>\"\"\" The Pakistani police came to our house and...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2091</th>\n",
       "      <td>10464</td>\n",
       "      <td>@@19612634</td>\n",
       "      <td>disabled</td>\n",
       "      <td>ie</td>\n",
       "      <td>\"When Marie O'Donoghue went looking for a spec...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2092</th>\n",
       "      <td>10465</td>\n",
       "      <td>@@14297363</td>\n",
       "      <td>women</td>\n",
       "      <td>lk</td>\n",
       "      <td>\"Sri Lankan norms and culture inhibit women fr...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2093</th>\n",
       "      <td>10466</td>\n",
       "      <td>@@70091353</td>\n",
       "      <td>vulnerable</td>\n",
       "      <td>ph</td>\n",
       "      <td>He added that the AFP will continue to bank on...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2094 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     par_id      art_id     keyword country  \\\n",
       "0      4046  @@14767805    hopeless      us   \n",
       "1      1279   @@7896098     refugee      ng   \n",
       "2      8330  @@17252299     refugee      ng   \n",
       "3      4063   @@3002894     in-need      ie   \n",
       "4      4089  @@25597822    homeless      pk   \n",
       "...     ...         ...         ...     ...   \n",
       "2089  10462  @@22092971    homeless      gh   \n",
       "2090  10463   @@4676355     refugee      pk   \n",
       "2091  10464  @@19612634    disabled      ie   \n",
       "2092  10465  @@14297363       women      lk   \n",
       "2093  10466  @@70091353  vulnerable      ph   \n",
       "\n",
       "                                                   text  label orig_label  \n",
       "0     We also know that they can benefit by receivin...      1          3  \n",
       "1     Pope Francis washed and kissed the feet of Mus...      1          4  \n",
       "2     Many refugees do n't want to be resettled anyw...      1          2  \n",
       "3     \"Budding chefs , like \"\" Fred \"\" , \"\" Winston ...      1          4  \n",
       "4     \"In a 90-degree view of his constituency , one...      1          3  \n",
       "...                                                 ...    ...        ...  \n",
       "2089  The sad spectacle , which occurred on Saturday...      0          0  \n",
       "2090  \"\"\" The Pakistani police came to our house and...      0          0  \n",
       "2091  \"When Marie O'Donoghue went looking for a spec...      0          0  \n",
       "2092  \"Sri Lankan norms and culture inhibit women fr...      0          1  \n",
       "2093  He added that the AFP will continue to bank on...      0          0  \n",
       "\n",
       "[2094 rows x 7 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from dont_patronize_me import DontPatronizeMe\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "dpm = DontPatronizeMe('data', '.')\n",
    "dpm.load_task1(\"test.tsv\")\n",
    "\n",
    "data = dpm.train_task1_df\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'disabled',\n",
       "  'homeless',\n",
       "  'hopeless',\n",
       "  'immigrant',\n",
       "  'in-need',\n",
       "  'migrant',\n",
       "  'poor-families',\n",
       "  'refugee',\n",
       "  'vulnerable',\n",
       "  'women'},\n",
       " {'au',\n",
       "  'bd',\n",
       "  'ca',\n",
       "  'gb',\n",
       "  'gh',\n",
       "  'hk',\n",
       "  'ie',\n",
       "  'in',\n",
       "  'jm',\n",
       "  'ke',\n",
       "  'lk',\n",
       "  'my',\n",
       "  'ng',\n",
       "  'nz',\n",
       "  'ph',\n",
       "  'pk',\n",
       "  'sg',\n",
       "  'tz',\n",
       "  'us',\n",
       "  'za'})"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keywords = set(data.keyword.to_list())\n",
    "countries = set(data.country.to_list())\n",
    "keywords, countries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "homeless: 29\n",
      "hopeless: 26\n",
      "migrant: 5\n",
      "disabled: 14\n",
      "women: 14\n",
      "vulnerable: 20\n",
      "refugee: 13\n",
      "in-need: 33\n",
      "immigrant: 7\n",
      "poor-families: 38\n",
      "lk: 9\n",
      "sg: 4\n",
      "ph: 15\n",
      "tz: 11\n",
      "us: 10\n",
      "ng: 15\n",
      "in: 7\n",
      "au: 5\n",
      "gh: 12\n",
      "ie: 10\n",
      "ke: 13\n",
      "bd: 7\n",
      "pk: 14\n",
      "gb: 9\n",
      "my: 8\n",
      "nz: 12\n",
      "hk: 9\n",
      "za: 11\n",
      "ca: 8\n",
      "jm: 10\n"
     ]
    }
   ],
   "source": [
    "data = data[data.label==1]\n",
    "# posdf\n",
    "\n",
    "for k in keywords:\n",
    "  print(f\"{k}: {len(data[data.keyword==k])}\")\n",
    "\n",
    "for k in countries:\n",
    "  print(f\"{k}: {len(data[data.country==k])}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "833607f6c7d60423acfa2c6660ca707b43172f7111d523e0f2b14146b62a227c"
  },
  "kernelspec": {
   "display_name": "Python 3.8.0 ('ic-nlp')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
